nys_fiscal_stress_analysis/processpdf.py at master · hhroc/nys_fiscal_stress_analysis · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
import json

if __name__ == '__main__':

    """

    The tex file is in the following format (repeated n-times).

    name
    class
    county
    region
    fiscal year end
    type of stress
    fiscal score
    environmental rating
    environmental score
    snapshot date

    Note: for some schools the 'type of stress' field has a value
          of 'Not filed' or 'Inconclusive'.  These need to be
          adressed as the next four fields will be missing after that.

    """

    # there are 10 fields in the doc
    FIELDCOUNT = 10

    # decoded text file
    filename = 'decoded.txt'

    print "Reading in file ..."

    with open(filename,'r') as f:
        contents = f.read()

    print "Pre-Processing file ..."

    # handle not filed and inconclusive cases
    contents = contents.replace('Not filed\n','Not filed\nN/A\nN/A\nN/A\n')
    contents = contents.replace('Inconclusive\n','Inconclusive\nN/A\nN/A\nN/A\n')
    contents = contents.replace('--------\n','') # page break

    # get each line
    lines = contents.split('\n')

    print "Processing file ..."

    districts = []
    for i in range(0,len(lines)/FIELDCOUNT):
        j = i * FIELDCOUNT
        district = {
            'name': lines[j+0],
            'class': lines[j+1],
            'county': lines[j+2],
            'region': lines[j+3],
            'fiscal_year_end': lines[j+4],
            'type_of_stress': lines[j+5],
            'fiscal_score': lines[j+6],
            'environmental_rating': lines[j+7],
            'environmental_score': lines[j+8],
            'snapshot_date': lines[j+9],
        }
        districts.append(district)

    print "Writing out json file ..."

    with open('districts.json','w') as f:
        f.write(json.dumps(districts))

    print "Writing out csv file ..."

    with open('districts.csv','w') as f:
        headers = 'name,' + \
                  'class,' + \
                  'county,' + \
                  'region,' + \
                  'fiscal year end,' + \
                  'type of stress,' + \
                  'fiscal score (%),' + \
                  'environmental rating,' + \
                  'environmental score (%),' + \
                  'snapshot date,\n'
        f.write(headers)

        for district in districts:

            fiscalscore = ''
            if district['fiscal_score'] != 'N/A':
                fiscalscore = str(float(district['fiscal_score'].replace('%',''))/100.0)

            environmentalscore = ''
            if district['environmental_score'] != 'N/A':
                environmentalscore = str(float(district['environmental_score'].replace('%',''))/100.0)

            line = '' + \
                district['name'] + ',' + \
                district['class'] + ',' + \
                district['county'] + ',' + \
                district['region'] + ',' + \
                district['fiscal_year_end'] + ',' + \
                district['type_of_stress'] + ',' + \
                fiscalscore + ',' + \
                district['environmental_rating'] + ',' + \
                environmentalscore + ',' + \
                district['snapshot_date'] + ',\n'

            f.write(line)

    print "Done."