From dbe2e14e062c0d9fc9bc9767c8f3c23bb11573be Mon Sep 17 00:00:00 2001
From: Lukas Zilka <lukas@zilka.me>
Date: Thu, 12 Mar 2015 17:59:31 +0100
Subject: [PATCH] Proposing changes to alex.utils & cleaning up.

---
 .../data/convert_idos_stops.py                |   4 +-
 .../data/expand_stops.py                      |   8 +-
 .../PublicTransportInfoCS/preprocessing.py    |   1 -
 alex/tests/test_tts_flite_en.py               |   3 -
 alex/tests/test_tts_google_cs.py              |   3 -
 alex/tests/test_tts_google_en.py              |   3 -
 alex/utils/audio_play.py                      |  23 ---
 alex/utils/cache.py                           |   3 +
 alex/utils/caminfodb.py                       |  44 -----
 alex/utils/config.py                          |   1 +
 alex/utils/cuda.py                            |  48 -----
 alex/utils/czech_stemmer.py                   | 173 ------------------
 alex/utils/enums.py                           |  11 --
 alex/utils/env.py                             |   2 +
 alex/utils/exceptions.py                      |   3 +-
 alex/utils/exdec.py                           |   1 +
 alex/utils/fs.py                              | 147 +--------------
 alex/utils/htk.py                             |   2 +-
 alex/utils/interface.py                       |  21 ---
 alex/utils/lattice.py                         |   1 +
 alex/utils/mfcc.py                            |   1 +
 alex/utils/mproc.py                           |   1 +
 alex/utils/parsers.py                         |  70 -------
 alex/utils/procname.py                        |   2 +-
 alex/utils/rdb.py                             |  27 ---
 alex/utils/sessionlogger.py                   |   1 +
 alex/utils/text.py                            |   1 +
 alex/utils/token.py                           |   1 +
 alex/utils/ui.py                              |   1 +
 alex/utils/various.py                         |  28 +--
 30 files changed, 27 insertions(+), 608 deletions(-)
 delete mode 100644 alex/utils/audio_play.py
 delete mode 100644 alex/utils/caminfodb.py
 delete mode 100644 alex/utils/cuda.py
 delete mode 100644 alex/utils/czech_stemmer.py
 delete mode 100644 alex/utils/enums.py
 delete mode 100644 alex/utils/interface.py
 delete mode 100644 alex/utils/parsers.py
 delete mode 100644 alex/utils/rdb.py

diff --git a/alex/applications/PublicTransportInfoCS/data/convert_idos_stops.py b/alex/applications/PublicTransportInfoCS/data/convert_idos_stops.py
index 1518fdc7..07383d0b 100755
--- a/alex/applications/PublicTransportInfoCS/data/convert_idos_stops.py
+++ b/alex/applications/PublicTransportInfoCS/data/convert_idos_stops.py
@@ -28,7 +28,7 @@
 import sys
 import re
 from add_cities_to_stops import load_list, get_city_for_stop
-from alex.utils.various import remove_dups_stable
+from alex.utils.various import list_remove_duplicates_keep_ordering
 from alex.components.nlg.tools.cs import word_for_number
 if __name__ == '__main__':
     import autopath
@@ -463,7 +463,7 @@ def expand_abbrevs(stop_name):
         try:
             # replacement variants
             if type(repls) == list:
-                variants = list(remove_dups_stable([regex.sub(repl, var)
+                variants = list(list_remove_duplicates_keep_ordering([regex.sub(repl, var)
                                                     for repl in repls for var in variants]))
             # just a single replacement
             else:
diff --git a/alex/applications/PublicTransportInfoCS/data/expand_stops.py b/alex/applications/PublicTransportInfoCS/data/expand_stops.py
index 0aa46c4a..061eba40 100755
--- a/alex/applications/PublicTransportInfoCS/data/expand_stops.py
+++ b/alex/applications/PublicTransportInfoCS/data/expand_stops.py
@@ -28,7 +28,7 @@
 from ufal.morphodita import Tagger, Forms, TaggedLemmas, TokenRanges, Morpho, TaggedLemmasForms
 
 from alex.utils.config import online_update
-from alex.utils.various import remove_dups_stable
+from alex.utils.various import list_remove_duplicates_keep_ordering
 from getopt import getopt
 
 
@@ -131,7 +131,7 @@ def load_file(self, fname):
                 if line.startswith('#'):  # skip comments
                     continue
                 stop, variants = self.parse_line(line)
-                self.stops[stop] = list(remove_dups_stable(variants + self.stops[stop]))
+                self.stops[stop] = list(list_remove_duplicates_keep_ordering(variants + self.stops[stop]))
 
     def expand_file(self, fname):
         """Load a list of stops from a file and expand it."""
@@ -156,9 +156,9 @@ def expand_file(self, fname):
                             prev_tag = word[2]
                         # use all possible combinations if there are more variants for this case
                         inflected = map(self.__postprocess_func,
-                                        remove_dups_stable([' '.join(var)
+                                        list_remove_duplicates_keep_ordering([' '.join(var)
                                                             for var in itertools.product(*forms)]))
-                        self.stops[stop] = list(remove_dups_stable(self.stops[stop] + inflected))
+                        self.stops[stop] = list(list_remove_duplicates_keep_ordering(self.stops[stop] + inflected))
                 ctr += 1
                 if ctr % 1000 == 0:
                     print >> sys.stderr, '.',
diff --git a/alex/applications/PublicTransportInfoCS/preprocessing.py b/alex/applications/PublicTransportInfoCS/preprocessing.py
index 96c7eaaf..c9a009fc 100644
--- a/alex/applications/PublicTransportInfoCS/preprocessing.py
+++ b/alex/applications/PublicTransportInfoCS/preprocessing.py
@@ -6,7 +6,6 @@
 
 from alex.components.slu.base import SLUPreprocessing
 from alex.components.asr.utterance import Utterance
-from alex.utils.czech_stemmer import cz_stem
 from alex.components.nlg.template import TemplateNLGPreprocessing
 from alex.components.nlg.tools.cs import word_for_number
 import re
diff --git a/alex/tests/test_tts_flite_en.py b/alex/tests/test_tts_flite_en.py
index 1eefb111..6a675dac 100755
--- a/alex/tests/test_tts_flite_en.py
+++ b/alex/tests/test_tts_flite_en.py
@@ -40,6 +40,3 @@
 
     print 'saving the TTS audio in ./tmp/flite_tts.wav'
     audio.save_wav(cfg, './tmp/flite_tts.wav', wav)
-
-    print 'playing audio'
-    audio.play(cfg, wav)
diff --git a/alex/tests/test_tts_google_cs.py b/alex/tests/test_tts_google_cs.py
index de2cb4b6..456d6c72 100755
--- a/alex/tests/test_tts_google_cs.py
+++ b/alex/tests/test_tts_google_cs.py
@@ -39,6 +39,3 @@
 
     print 'saving the TTS audio in ./tmp/google_tts.wav'
     audio.save_wav(cfg, './tmp/google_tts.wav', wav)
-
-    print 'playing audio'
-    audio.play(cfg, wav)
diff --git a/alex/tests/test_tts_google_en.py b/alex/tests/test_tts_google_en.py
index 4624480b..c2e74b37 100755
--- a/alex/tests/test_tts_google_en.py
+++ b/alex/tests/test_tts_google_en.py
@@ -39,6 +39,3 @@
 
     print 'saving the TTS audio in ./tmp/google_tts.wav'
     audio.save_wav(cfg, './tmp/google_tts.wav', wav)
-
-    print 'playing audio'
-    audio.play(cfg, wav)
diff --git a/alex/utils/audio_play.py b/alex/utils/audio_play.py
deleted file mode 100644
index 257d6a99..00000000
--- a/alex/utils/audio_play.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import pyaudio
-
-
-
-def play(cfg, wav):
-    # open the audio device
-    p = pyaudio.PyAudio()
-
-    chunk = 160
-    # open stream
-    stream = p.open(format=p.get_format_from_width(pyaudio.paInt32),
-                    channels=1,
-                    rate=cfg['Audio']['sample_rate'],
-                    output=True,
-                    frames_per_buffer=chunk)
-
-    wav = various.split_to_bins(wav, chunk)
-    for w in wav:
-        stream.write(w)
-
-    stream.stop_stream()
-    stream.close()
-    p.terminate()
\ No newline at end of file
diff --git a/alex/utils/cache.py b/alex/utils/cache.py
index 7853516d..272b5a61 100644
--- a/alex/utils/cache.py
+++ b/alex/utils/cache.py
@@ -16,6 +16,7 @@
 from heapq import nsmallest
 from operator import itemgetter
 
+# TODO: Move this to config. This is evil.
 persistent_cache_directory = '~/.alex_persistent_cache'
 
 
@@ -229,6 +230,8 @@ def wrapper(*args, **kwds):
 
     return decorator
 
+
+# TODO: Fix.
 persistent_cache_directory = os.path.expanduser(persistent_cache_directory)
 if not os.path.exists(persistent_cache_directory):
     os.makedirs(persistent_cache_directory)
diff --git a/alex/utils/caminfodb.py b/alex/utils/caminfodb.py
deleted file mode 100644
index 9f0c1557..00000000
--- a/alex/utils/caminfodb.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from alex.utils.parsers import CamTxtParser
-
-class CamInfoDb(object):
-    def __init__(self, db_path):
-        ctp = CamTxtParser(lower=True)
-        self.data = ctp.parse(db_path)
-
-    def matches(self, rec, query):
-        for key, value in query.items():
-            if not key in rec or not value in rec.get(key):
-                return False
-
-        return True
-
-    def get_by_id(self, rec_id):
-        for rec in self.data:
-            if rec_id in rec.get('id'):
-                return rec
-
-        return None
-
-    def get_matching(self, query):
-        res = []
-        for rec in self.data:
-            if self.matches(rec, query):
-                res += [rec]
-
-        return res
-
-    def get_possible_values(self):
-        res = []
-        for item in self.data:
-            for val_item in item.values():
-                res += val_item
-        return set(res)
-
-    def get_slots(self):
-        slots = set()
-        for item in self.data:
-            for key in item.keys():
-                slots.add(key)
-        return slots
-
-
diff --git a/alex/utils/config.py b/alex/utils/config.py
index b1b6dd6a..d3364a1a 100644
--- a/alex/utils/config.py
+++ b/alex/utils/config.py
@@ -24,6 +24,7 @@
 
 config = None
 
+# TODO: Move this to config.
 online_update_server = "https://vystadial.ms.mff.cuni.cz/download/alex/"
 
 
diff --git a/alex/utils/cuda.py b/alex/utils/cuda.py
deleted file mode 100644
index 24a79067..00000000
--- a/alex/utils/cuda.py
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-
-def cudasolve(A, b, tol=1e-3, normal=False, regA = 1.0, regI = 0.0):
-    """ Conjugate gradient solver for dense system of linear equations.
-
-        Ax = b
-
-        Returns: x = A^(-1)b
-
-        If the system is normal, then it solves
-
-        (regA*A'A +regI*I)x= b
-
-        Returns: x = (A'A +reg*I)^(-1)b
-    """
-
-    N = len(b)
-    b = b.reshape((N,1))
-    b_norm = culinalg.norm(b)
-    x = b.copy()
-    if not normal:
-        r = b - culinalg.dot(A,x)
-    else:
-        r = b - regA*culinalg.dot(A,culinalg.dot(A,x), transa='T') - regI*x
-    p = r.copy()
-    rsold = culinalg.dot(r,r, transa='T')[0][0].get()
-    for i in range(N):
-        if not normal:
-            Ap = culinalg.dot(A,p)
-        else:
-            Ap = regA*culinalg.dot(A,culinalg.dot(A,p), transa='T') + regI*p
-
-        pAp = culinalg.dot(p, Ap, transa='T')[0][0].get()
-        alpha = rsold / pAp
-
-        x += alpha*p
-        r -= alpha*Ap
-        rsnew = culinalg.dot(r,r, transa='T')[0][0].get()
-
-        if math.sqrt(rsnew)/b_norm < tol:
-            break
-        else:
-            p = r + (rsnew/rsold)*p
-            rsold = rsnew
-
-    return x.reshape(N)
diff --git a/alex/utils/czech_stemmer.py b/alex/utils/czech_stemmer.py
deleted file mode 100644
index ee5bb462..00000000
--- a/alex/utils/czech_stemmer.py
+++ /dev/null
@@ -1,173 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-''' Czech stemmer
-Copyright © 2010 Luís Gomes <luismsgomes@gmail.com>.
-
-Ported from the Java implementation available at:
-    http://members.unine.ch/jacques.savoy/clef/index.html
-
-'''
-import re
-import sys
-
-def cz_stem_word(word, aggressive=False):
-    if not re.match(u"^\\w+$", word):
-        return word
-    if not word.islower() and not word.istitle() and not word.isupper():
-        return word
-    s = word.lower() # all our pattern matching is done in lowercase
-    s = _remove_case(s)
-    s = _remove_possessives(s)
-    if aggressive:
-        s = _remove_comparative(s)
-        s = _remove_diminutive(s)
-        s = _remove_augmentative(s)
-        s = _remove_derivational(s)
-    if word.isupper():
-        return s.upper()
-    if word.istitle():
-        return s.title()
-    return s
-
-def cz_stem(l, aggressive=False):
-    if isinstance(l, str) or isinstance(l, unicode):
-        return cz_stem_word(l)
-    else:
-        return [cz_stem(w) for w in l if w]
-
-def _remove_case(word):
-    if len(word) > 7 and word.endswith(u"atech"):
-        return word[:-5]
-    if len(word) > 6:
-        if word.endswith(u"ětem"):
-            return _palatalise(word[:-3])
-        if word.endswith(u"atům"):
-            return word[:-4]
-    if len(word) > 5:
-        if word[-3:] in {u"ech", u"ich", u"ích", u"ého", u"ěmi", u"emi", u"ému",
-                         u"ete", u"eti", u"iho", u"ího", u"ími", u"imu"}:
-            return _palatalise(word[:-2])
-        if word[-3:] in {u"ách", u"ata", u"aty", u"ých", u"ama", u"ami",
-                         u"ové", u"ovi", u"ými"}:
-            return word[:-3]
-    if len(word) > 4:
-        if word.endswith(u"em"):
-            return _palatalise(word[:-1])
-        if word[-2:] in {u"es", u"ém", u"ím"}:
-            return _palatalise(word[:-2])
-        if word[-2:] in {u"ům", u"at", u"ám", u"os", u"us", u"ým", u"mi", u"ou"}:
-            return word[:-2]
-    if len(word) > 3:
-        if word[-1] in u"eiíě":
-            return _palatalise(word)
-        if word[-1] in u"uyůaoáéý":
-            return word[:-1]
-    return word
-
-def _remove_possessives(word):
-    if len(word) > 5:
-        if word[-2:] in {u"ov", u"ův"}:
-            return word[:-2]
-        if word.endswith(u"in"):
-            return _palatalise(word[:-1])
-    return word
-
-def _remove_comparative(word):
-    if len(word) > 5:
-        if word[-3:] in {u"ejš", u"ějš"}:
-            return _palatalise(word[:-2])
-    return word
-
-def _remove_diminutive(word):
-    if len(word) > 7 and word.endswith(u"oušek"):
-        return word[:-5]
-    if len(word) > 6:
-        if word[-4:] in {u"eček", u"éček", u"iček", u"íček", u"enek", u"ének",
-                         u"inek", u"ínek"}:
-            return _palatalise(word[:-3])
-        if word[-4:] in {u"áček", u"aček", u"oček", u"uček", u"anek", u"onek",
-                         u"unek", u"ánek"}:
-            return _palatalise(word[:-4])
-    if len(word) > 5:
-        if word[-3:] in {u"ečk", u"éčk", u"ičk", u"íčk", u"enk", u"énk",
-                         u"ink", u"ínk"}:
-            return _palatalise(word[:-3])
-        if word[-3:] in {u"áčk", u"ačk", u"očk", u"učk", u"ank", u"onk",
-                         u"unk", u"átk", u"ánk", u"ušk"}:
-            return word[:-3]
-    if len(word) > 4:
-        if word[-2:] in {u"ek", u"ék", u"ík", u"ik"}:
-            return _palatalise(word[:-1])
-        if word[-2:] in {u"ák", u"ak", u"ok", u"uk"}:
-            return word[:-1]
-    if len(word) > 3 and word[-1] == u"k":
-        return word[:-1]
-    return word
-
-def _remove_augmentative(word):
-    if len(word) > 6 and word.endswith(u"ajzn"):
-        return word[:-4]
-    if len(word) > 5 and word[-3:] in {u"izn", u"isk"}:
-        return _palatalise(word[:-2])
-    if len(word) > 4 and word.endswith(u"ák"):
-        return word[:-2]
-    return word
-
-def _remove_derivational(word):
-    if len(word) > 8 and word.endswith(u"obinec"):
-        return word[:-6]
-    if len(word) > 7:
-        if word.endswith(u"ionář"):
-            return _palatalise(word[:-4])
-        if word[-5:] in {u"ovisk", u"ovstv", u"ovišt", u"ovník"}:
-            return word[:-5]
-    if len(word) > 6:
-        if word[-4:] in {u"ásek", u"loun", u"nost", u"teln", u"ovec", u"ovík",
-                         u"ovtv", u"ovin", u"štin"}:
-            return word[:-4]
-        if word[-4:] in {u"enic", u"inec", u"itel"}:
-            return _palatalise(word[:-3])
-    if len(word) > 5:
-        if word.endswith(u"árn"):
-            return word[:-3]
-        if word[-3:] in {u"ěnk", u"ián", u"ist", u"isk", u"išt", u"itb", u"írn"}:
-            return _palatalise(word[:-2])
-        if word[-3:] in {u"och", u"ost", u"ovn", u"oun", u"out", u"ouš",
-                         u"ušk", u"kyn", u"čan", u"kář", u"néř", u"ník",
-                         u"ctv", u"stv"}:
-            return word[:-3]
-    if len(word) > 4:
-        if word[-2:] in {u"áč", u"ač", u"án", u"an", u"ář", u"as"}:
-            return word[:-2]
-        if word[-2:] in {u"ec", u"en", u"ěn", u"éř", u"íř", u"ic", u"in", u"ín",
-                         u"it", u"iv"}:
-            return _palatalise(word[:-1])
-        if word[-2:] in {u"ob", u"ot", u"ov", u"oň", u"ul", u"yn", u"čk", u"čn",
-                         u"dl", u"nk", u"tv", u"tk", u"vk"}:
-            return word[:-2]
-    if len(word) > 3 and word[-1] in u"cčklnt":
-        return word[:-1]
-    return word
-
-def _palatalise(word):
-    if word[-2:] in {u"ci", u"ce", u"či", u"če"}:
-        return word[:-2] + u"k"
-
-    if word[-2:] in {u"zi", u"ze", u"ži", u"že"}:
-        return word[:-2] + u"h"
-
-    if word[-3:] in {u"čtě", u"čti", u"čtí"}:
-        return word[:-3] + u"ck"
-
-    if word[-3:] in {u"ště", u"šti", u"ští"}:
-        return word[:-3] + u"sk"
-    return word[:-1]
-
-if __name__ == '__main__':
-    if len(sys.argv) != 2 or sys.argv[1] not in (u"light", u"aggressive"):
-        sys.exit(u"usage: {} light|aggressive".format(sys.argv[0]))
-    aggressive = sys.argv[1] == u"aggressive"
-    for line in sys.stdin:
-        print u" ".join([cz_stem(word, aggressive=aggressive)
-                for word in line.split()])
diff --git a/alex/utils/enums.py b/alex/utils/enums.py
deleted file mode 100644
index 7580f6a8..00000000
--- a/alex/utils/enums.py
+++ /dev/null
@@ -1,11 +0,0 @@
-
-def enum(*sequential, **named):
-    """Useful for creating enumerations.
-
-    e.g.:
-    DialogueType = enum(deterministic=0, statistical=1, mix=2)"""
-
-    enums = dict(zip(sequential, range(len(sequential))), **named)
-    reverse = dict((value, key) for key, value in enums.iteritems())
-    enums['reverse_mapping'] = reverse
-    return type('Enum', (), enums)
\ No newline at end of file
diff --git a/alex/utils/env.py b/alex/utils/env.py
index f031539b..137da469 100644
--- a/alex/utils/env.py
+++ b/alex/utils/env.py
@@ -1,5 +1,7 @@
+# TODO: Put this into the __init__.py of alex.utils.
 import os
 
+
 def root():
     """
     Finds the root of the project and return it as string.
diff --git a/alex/utils/exceptions.py b/alex/utils/exceptions.py
index f677c439..dd6fa7eb 100644
--- a/alex/utils/exceptions.py
+++ b/alex/utils/exceptions.py
@@ -1,3 +1,4 @@
+# TODO: Move the exceptions to the objects that raise them.
 from alex import AlexException
 
 
@@ -8,8 +9,6 @@ class ConfigException(AlexException):
 class SessionLoggerException(AlexException):
     pass
 
-class SessionLoggerException(AlexException):
-    pass
 
 class SessionClosedException(AlexException):
     pass
diff --git a/alex/utils/exdec.py b/alex/utils/exdec.py
index 6401ace8..a356afe4 100644
--- a/alex/utils/exdec.py
+++ b/alex/utils/exdec.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 #
 # pylint: disable-msg=E1103
+# TODO: Move directly to alex.utils.
 
 import functools
 import traceback
diff --git a/alex/utils/fs.py b/alex/utils/fs.py
index 33a59970..392b9eac 100644
--- a/alex/utils/fs.py
+++ b/alex/utils/fs.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 # This code is PEP8-compliant. See http://www.python.org/dev/peps/pep-0008.
+# TODO: Move this directly to alex.utils.
 """
 Filesystem utility functions.
 
@@ -177,149 +178,3 @@ def _find_ignorefunc(dir_, glob_, mindepth, maxdepth, ignore_globs=list(),
             visited.update(more_visited)
     # Return.
     return matched, visited
-
-
-class GrepFilter(Process):
-
-    def __init__(self, stdin, stdout, breakchar='\n'):
-        Process.__init__(self)
-        self.stdin = stdin
-        self.stdout = stdout
-        self.buf = ''
-        self.breakchar = breakchar
-        self.listeners = list()  # :: [(regex, callback)]
-        self.valid = list()      # :: [?is_listener_valid]
-        self.closed = False
-
-    def add_listener(self, regex, callback):
-        """
-        Adds a listener to the output strings.
-
-        Arguments:
-            regex -- the compiled regular expression to look for
-                (`regex.search') in any piece of output
-            callback -- a callable that is invoked for output where `regex' was
-                found.  This will be called like this:
-
-                    outputting &= callback(output_unicode_str)
-
-                That means, callback should take the unicode string argument
-                containing what would have been output and return a boolean
-                value which is True iff outputting should stop.
-
-        Returns the index of the listener for later reference.
-
-        """
-        if all(self.valid):
-            idx = len(self.listeners)
-            self.listeners.append((regex, callback))
-            self.valid.append(True)
-        else:
-            idx = self.valid.index(False)
-            self.listeners[idx] = (regex, callback)
-            self.valid[idx] = True
-        return idx
-
-    def remove_listener(self, listener_idx):
-        if listener_idx >= len(self.listeners):
-            raise IndexError('Listener index out of bounds.')
-        if listener_idx == len(self.listeners) - 1:
-            del self.listeners[-1]
-            del self.valid[-1]
-        else:
-            self.valid[listener_idx] = False
-
-    def flush(self, force=True):
-        flushing = False
-        indata = ''
-        while True:
-            ready_inputs, _, _ = select((self.stdin, ), tuple(), tuple(), 0)
-            if not ready_inputs:
-                break
-
-            indata = self.stdin.read()
-            if not indata:
-                break
-
-            flushing |= self.breakchar in indata
-            self.buf += indata
-            if DEBUG:
-                print "grep: stdin received:", indata
-                print "flushing:", flushing
-                print "self.buf:", self.buf
-            if force and self.buf and self.buf[-1] != self.breakchar:
-                self.buf += self.breakchar
-                flushing = True
-        if flushing:
-            chunks = self.buf.split(self.breakchar)
-            if DEBUG:
-                print "chunks:", chunks
-            self.buf = chunks[-1]
-            del chunks[-1]
-            map(self.write, chunks)
-
-    def write(self, unistr):
-        # DEBUG
-        if DEBUG:
-            print
-            print 'Outputting "{}"'.format(unistr)
-
-        outputting = True
-        for regex, callback in self.listeners:
-            if regex.search(unistr):
-                outputting &= callback(unistr)
-        # DEBUG
-        if DEBUG:
-            print 'outputting={}'.format(outputting)
-            print
-
-        if outputting:
-            if self.closed:
-                raise Exception('The output stream has already been closed!')
-            self.stdout.write(unistr + self.breakchar)
-        else:
-            self.closed = True
-
-    def run(self):
-        while True:
-            self.flush(False)
-            sleep(.1)
-
-
-# TODO Fix this and remove this file from nose2.cfg.
-# XXX This is very provisional.
-def test_grep_filter():
-    # from StringIO import StringIO
-    from subprocess import Popen, PIPE
-    import re
-    import sys
-
-    devnull = open('/dev/null', 'a+')
-    producer = Popen(
-        ['head', '-n', '3', __file__], stdout=PIPE, stderr=devnull)
-
-    # sio = StringIO()
-    sio = sys.stdout
-    gio = GrepFilter(producer.stdout, sio)
-
-    def gfunc(outstr):
-        print "(II) match found:", outstr
-        # return False
-        return True
-    rx = re.compile('ho', re.I)
-    gio.add_listener(rx, gfunc)
-    gio.start()
-
-    if DEBUG:
-        print producer.stdout
-        sleep(1.)
-        from pprint import pprint
-        pprint(producer.__dict__.items())
-
-    gio.flush()
-    gio.terminate()
-
-
-# XXX This is very provisional.
-if __name__ == "__main__":
-    test_grep_filter()
diff --git a/alex/utils/htk.py b/alex/utils/htk.py
index b7a7b343..e74cc8ee 100644
--- a/alex/utils/htk.py
+++ b/alex/utils/htk.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-#
+# TODO: Move this to alex/utils/asr/.
 
 import numpy
 import re
diff --git a/alex/utils/interface.py b/alex/utils/interface.py
deleted file mode 100644
index e085ea23..00000000
--- a/alex/utils/interface.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import inspect
-
-def interface_method(f):
-    f.abstract = True
-    return f
-
-class Interface(object):
-    def __new__(cls, *args, **kwargs):
-        res = super(Interface, cls).__new__(cls, *args, **kwargs)
-
-        missing_methods = []
-        for method in inspect.getmembers(res, predicate=inspect.ismethod):
-            if getattr(method[1], 'abstract', False):
-                missing_methods += [method[0]]
-
-        if len(missing_methods) > 0:
-            raise Exception("Class %s is missing these interface methods: %s" %\
-                            (cls.__name__,", ".join((missing_methods))))
-
-        return res
-
diff --git a/alex/utils/lattice.py b/alex/utils/lattice.py
index 5b98d9e2..5c149615 100755
--- a/alex/utils/lattice.py
+++ b/alex/utils/lattice.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # This code is PEP8-compliant. See http://www.python.org/dev/peps/pep-0008.
+# TODO: Move this to alex/components/asr.
 '''
 This module stores functionality which requires pysft module installed.
 See https://github.com/UFAL-DSG/pyfst
diff --git a/alex/utils/mfcc.py b/alex/utils/mfcc.py
index 8f890ea6..a1cf0598 100644
--- a/alex/utils/mfcc.py
+++ b/alex/utils/mfcc.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+# TODO: Move this to alex/utils/asr/.
 
 import numpy as np
 
diff --git a/alex/utils/mproc.py b/alex/utils/mproc.py
index 5bb6687b..b893d710 100644
--- a/alex/utils/mproc.py
+++ b/alex/utils/mproc.py
@@ -143,6 +143,7 @@ def get_instance_id(self):
         return InstanceID.instance_id.value
 
 
+# TODO: Move this class to alex.utils.logger.
 class SystemLogger(object):
     """
     This is a multiprocessing-safe logger.  It should be used by all components in Alex.
diff --git a/alex/utils/parsers.py b/alex/utils/parsers.py
deleted file mode 100644
index 7eae4a4c..00000000
--- a/alex/utils/parsers.py
+++ /dev/null
@@ -1,70 +0,0 @@
-import re
-import sys
-from collections import defaultdict
-
-class CamTxtParser(object):
-    """Parser of files of the following format:
-    <<BOF>>
-    [record]
-
-    [record]
-
-    ...
-    <<EOF>>
-
-    where [record] has the following format:
-
-    <<[record]>>
-    [property name]([property value])
-    <</[record]>>
-
-    [property name] and [property value] are arbitrary strings
-
-    Any " or ' characters are stripped from the beginning and end
-    of each [property value]."""
-
-    line_expr = re.compile(r"^(?P<name>[^\s]*)\((?P<value>.*)\)$")
-
-    def __init__(self, lower=False):
-        self.lower = lower
-
-    def parse(self, f_obj):
-        """Parse the given file and return list of dictionaries with parsed values.
-
-        Arguments:
-        f_obj -- filename of file or file object to be parsed"""
-
-        if type(f_obj) is str:
-            f_obj = open(f_obj, 'r+b')
-
-        assert type(f_obj) is file
-
-        objs = []
-        n_obj = defaultdict(lambda: [])
-        blank = True
-
-        for ln in f_obj:
-            if self.lower:
-                ln = ln.lower()
-            ln = ln.strip()
-            if len(ln) == 0 and not blank:
-                objs += [n_obj]
-                n_obj = defaultdict(lambda: [])
-                blank = True
-
-            m_obj = self.line_expr.match(ln)
-            if m_obj is not None:
-                blank = False
-
-                try:
-                    key, value = m_obj.groups()
-                    value = value.strip('"').strip("'")
-                    n_obj[key] += [value]
-                except ValueError:
-                    print >>sys.stderr, 'ignoring', m_obj.groups()
-
-        return objs
-
-
-if __name__ == '__main__':
-    print CamTxtParser().parse("/xdisk/devel/vystadial/alex//applications/CamInfoRest/cued_data/CIRdbase_V7.txt")
diff --git a/alex/utils/procname.py b/alex/utils/procname.py
index dbbbeb22..baf01ccf 100644
--- a/alex/utils/procname.py
+++ b/alex/utils/procname.py
@@ -1,4 +1,4 @@
-
+# TODO: Move this directly to alex.utils.
 
 def set_proc_name(newname):
     from ctypes import cdll, byref, create_string_buffer
diff --git a/alex/utils/rdb.py b/alex/utils/rdb.py
deleted file mode 100644
index 94c81056..00000000
--- a/alex/utils/rdb.py
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env python
-
-import pdb
-import socket
-import sys
-
-class Rdb(pdb.Pdb):
-    def __init__(self, port=4446):
-        self.old_stdout = sys.stdout
-        self.old_stdin = sys.stdin
-        self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.skt.bind((socket.gethostname(), port))
-        self.skt.listen(1)
-        print 'listening'
-        (clientsocket, address) = self.skt.accept()
-        handle = clientsocket.makefile('rw')
-        pdb.Pdb.__init__(self, completekey='tab', stdin=handle, stdout=handle)
-        sys.stdout = sys.stdin = handle
-
-    def do_continue(self, arg):
-        sys.stdout = self.old_stdout
-        sys.stdin = self.old_stdin
-        self.skt.close()
-        self.set_continue()
-        return 1
-
-    do_c = do_cont = do_continue
\ No newline at end of file
diff --git a/alex/utils/sessionlogger.py b/alex/utils/sessionlogger.py
index f5c50557..e42f1309 100644
--- a/alex/utils/sessionlogger.py
+++ b/alex/utils/sessionlogger.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 # This code is mostly PEP8-compliant. See
 # http://www.python.org/dev/peps/pep-0008.
+# TODO: Import this in alex.utils.
 
 import multiprocessing
 import time
diff --git a/alex/utils/text.py b/alex/utils/text.py
index a13f466e..3b173e4f 100644
--- a/alex/utils/text.py
+++ b/alex/utils/text.py
@@ -16,6 +16,7 @@ def findall(text, char, start=0, end=-1):
         nextidx = text.find(char, nextidx + 1, end)
     return idxs
 
+
 def split_by_comma(text):
     parentheses = 0
     splitList = []
diff --git a/alex/utils/token.py b/alex/utils/token.py
index 742e878a..9bf075aa 100644
--- a/alex/utils/token.py
+++ b/alex/utils/token.py
@@ -1,3 +1,4 @@
+# TODO: Remove. It is not used.
 import urllib2
 
 
diff --git a/alex/utils/ui.py b/alex/utils/ui.py
index 88d94625..bb02e25d 100644
--- a/alex/utils/ui.py
+++ b/alex/utils/ui.py
@@ -6,6 +6,7 @@
 
 # Stolen and adapted from
 # http://stackoverflow.com/questions/566746/how-to-get-console-window-width-in-python.
+# TODO: Rename this to get_terminal_size and move directly to alex.utils.
 def getTerminalSize():
     """ Retrieves the size of the current terminal window.
 
diff --git a/alex/utils/various.py b/alex/utils/various.py
index 1b3c8e26..92413225 100644
--- a/alex/utils/various.py
+++ b/alex/utils/various.py
@@ -3,12 +3,14 @@
 # This code is PEP8-compliant. See http://www.python.org/dev/peps/pep-0008.
 #
 # pylint: disable-msg=E1101
+# TODO: Move directly to alex.utils.
 
 from collections import defaultdict
 import math
 import sys
 
 
+# TODO: Move this to alex.applications.utils.
 def split_to_bins(A, S=4):
     """Split the A array into bins of size N."""
     m, n = divmod(len(A), S)
@@ -60,31 +62,7 @@ def walk(self):
                     yield key, value
 
 
-def crop_to_finite(val):
-    if math.isinf(val):
-        if isinstance(val, int):
-            ret = sys.maxint
-        else:
-            ret = sys.float_info.max
-        return (-1 + 2 * (val > 0)) * ret
-    return val
-
-
-def group_by(objects, attrs):
-    """Groups `objects' by the values of their attributes `attrs'.
-
-    Returns a dictionary mapping from a tuple of attribute values to a list of
-    objects with those attribute values.
-
-    """
-    groups = dict()
-    for obj in objects:
-        key = tuple(getattr(obj, attr) for attr in attrs)
-        groups.setdefault(key, []).append(obj)
-    return groups
-
-
-def remove_dups_stable(l):
+def list_remove_duplicates_keep_ordering(l):
     """Remove duplicates from a list but keep the ordering.
 
     @return: Iterator over unique values in the list