DRST/seq_generators.py at master · saeliddp/DRST · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
import re
from constants import *
from viet_diacritic_mark import VietDiacriticMark, get_mark_viet, src_char_to_ix

syll_split_pattern = r'[\s\W]+'
# training sequences of characters with labels being diacritics
def char_diac_seq_generator(src_fpath, target_fpath):
  src = open(src_fpath, 'r', encoding='utf-8')
  target = open(target_fpath, 'r', encoding='utf-8')

  for src_line in src:
    target_line = target.readline()
    src_sylls = re.split(syll_split_pattern, src_line)
    target_sylls = re.split(syll_split_pattern, target_line)

    chars = []
    tags = []
    for src_syll, target_syll in zip(src_sylls, target_sylls):
      if src_syll and target_syll:
        chars.append(WORD_START)
        tags.append(VietDiacriticMark.NONE_NONE)
        if re.search(r'[0-9]', src_syll):
          chars.append(NUMERIC)
          tags.append(VietDiacriticMark.NONE_NONE)
        else:
          i = 0
          for src_char in src_syll:
            chars.append(src_char if src_char in src_char_to_ix else UNKNOWN)
            tags.append(get_mark_viet(target_syll[i]))
            i += 1

    yield chars, tags

  src.close()
  target.close()

# training sequences of characters with labels being characters
def char_char_seq_generator(src_fpath, target_fpath):
  src = open(src_fpath, 'r', encoding='utf-8')
  target = open(target_fpath, 'r', encoding='utf-8')

  for src_line in src:
    target_line = target.readline()
    src_sylls = re.split(syll_split_pattern, src_line)
    target_sylls = re.split(syll_split_pattern, target_line)

    chars = []
    tags = []
    for src_syll, target_syll in zip(src_sylls, target_sylls):
      if src_syll and target_syll:
        chars.append(WORD_START)
        tags.append(WORD_START)
        if re.search(r'[0-9]', src_syll):
          chars.append(NUMERIC)
          tags.append(NUMERIC)
        else:
          i = 0
          for src_char in src_syll:
            chars.append(src_char)
            tags.append(target_syll[i])
            i += 1

    yield chars, tags

  src.close()
  target.close()

# training sequences of syllables with labels being syllables
def syll_syll_seq_generator(src_fpath, target_fpath):
  src = open(src_fpath, 'r', encoding='utf-8')
  target = open(target_fpath, 'r', encoding='utf-8')

  for src_line in src:
    target_line = target.readline()
    src_sylls = re.split(syll_split_pattern, src_line)
    target_sylls = re.split(syll_split_pattern, target_line)

    src_sylls_final = [SENT_START]
    target_sylls_final = [SENT_START]
    for src_syll, target_syll in zip(src_sylls, target_sylls):
      if src_syll and target_syll:
        if re.search(r'[0-9]', src_syll):
          src_sylls_final.append(NUMERIC)
          target_sylls_final.append(NUMERIC)
        else:
          src_sylls_final.append(src_syll)
          target_sylls_final.append(target_syll)

    yield src_sylls_final, target_sylls_final

  src.close()
  target.close()