slearner/model_preprocessor_tests.py at master · CyEMI/slearner · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
import unittest
import os
import filecmp

from model_preprocess import ModelPreprocessor, BulkModelProcessor


class TestModelPreprocessor(unittest.TestCase):

    my_dir = os.path.dirname(os.path.realpath(__file__))

    def test_sampleModel(self):
        sys_loc = os.path.join(self.my_dir, 'testdata', 'sampleModel20.mdl')
        out_loc = os.path.join(self.my_dir, 'output')

        mp = ModelPreprocessor(sys_loc, out_loc, postprocess=False)
        result = mp.go(True)
        self.assertIsNone(result)

    def test_sampleModel_postprocess(self):
        sys_loc = os.path.join(self.my_dir, 'testdata', 'sampleModel20.mdl')
        out_loc = os.path.join(self.my_dir, 'output')

        mp = ModelPreprocessor(sys_loc, out_loc, postprocess=True)
        result = mp.go(True)
        self.assertIsNone(result)

    def test_smoke1(self):
        sys_loc = os.path.join(self.my_dir, 'testdata', 'sampleModel1.mdl')
        out_loc = os.path.join(self.my_dir, 'output')

        mp = ModelPreprocessor(sys_loc, out_loc, postprocess=False)
        result = mp.go(True)
        self.assertIsNone(result)

    def test_smoke1_postprocess(self):
        sys_loc = os.path.join(self.my_dir, 'testdata', 'sampleModel1.mdl')
        out_loc = os.path.join(self.my_dir, 'output')

        mp = ModelPreprocessor(sys_loc, out_loc, postprocess=True)
        mp.go(True)
        self.assertTrue(filecmp.cmp(os.path.join(self.my_dir, 'output', 'sampleModel1.mdl'),
                                    os.path.join(self.my_dir, 'testdata', '1pp.txt')
                                    , shallow=False), 'Model sampleModel1 mismatched')


class TestBulkModelPreprocessor(unittest.TestCase):

    my_dir = os.path.dirname(os.path.realpath(__file__))

    def test_smoke(self):
        """
        Converts all three model files from testdata folder and compare
        """
        sys_loc = os.path.join(self.my_dir, 'testdata')
        out_loc = os.path.join(self.my_dir, 'output')

        mp = BulkModelProcessor(sys_loc, out_loc, postprocess=False)
        mp.go(True)

        for i in (20, 21, 22,):
            o_loc = os.path.join(self.my_dir, 'output', 'sampleModel{}.mdl'.format(i))
            base_loc = os.path.join(self.my_dir, 'testdata', '{}ni.txt'.format(i))
            self.assertTrue(filecmp.cmp(o_loc, base_loc, shallow=False), 'Model sampleModel{} mismatched'.format(i))

        # does not work.. set order not reliable? #TODO construct set and compare
        # self.assertTrue(filecmp.cmp(os.path.join(self.my_dir, 'output', 'unique_keywords.txt'),
        #                 os.path.join(self.my_dir, 'testdata', '202122_uk.txt'), shallow=False),
        #                 'Unique Keywords text file mismatched')

    def test_smoke_postprocess(self):
        """
        Converts all three model files from testdata folder and compare
        """
        sys_loc = os.path.join(self.my_dir, 'testdata')
        out_loc = os.path.join(self.my_dir, 'output')

        mp = BulkModelProcessor(sys_loc, out_loc, postprocess=True)
        mp.go(True)

    def test_corpus(self):
        sys_loc = '/home/cyfuzz/workspace/explore/success'
        out_loc = '/home/cyfuzz/workspace/explore/processed'

        mp = BulkModelProcessor(sys_loc, out_loc, postprocess=False)
        mp.go(True)