From d36a4a04ceca74769fc6c8b0fe7584ab3ead71ae Mon Sep 17 00:00:00 2001
From: Serdar Tumgoren <zstumgoren@gmail.com>
Date: Fri, 2 Dec 2016 20:49:51 -0800
Subject: [PATCH 1/3] DC: Add debugger script that dumps raw bill json

---
 scripts/dc/dump_bill_json.py | 43 ++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 scripts/dc/dump_bill_json.py

diff --git a/scripts/dc/dump_bill_json.py b/scripts/dc/dump_bill_json.py
new file mode 100644
index 0000000000..e53c1d5071
--- /dev/null
+++ b/scripts/dc/dump_bill_json.py
@@ -0,0 +1,43 @@
+'''Dump JSON for a DC bill
+
+USAGE:
+
+    $ python dump_bill_json.py PR21-0316
+
+'''
+import json
+import pprint
+import sys
+
+import requests
+
+def main():
+    headers = {"Content-Type":"application/json"}
+    bill_url = "http://lims.dccouncil.us/_layouts/15/uploader/AdminProxy.aspx/GetPublicData"
+    try:
+        bill_id = sys.argv[1]
+    except IndexError:
+        msg = "\nERROR: You must supply a bill id! Example:\n\n\tpython {} PR21-0316\n".format(__file__)
+        sys.exit(msg)
+    bill_params = { "legislationId" : bill_id }
+    bill_info = requests.post(bill_url, headers=headers, data=json.dumps(bill_params))
+    bill_info = decode_json(bill_info.json()["d"])["data"]
+    leg_info = bill_info["Legislation"][0]
+    pprint.pprint(bill_info)
+
+def decode_json(stringy_json):
+    #the "json" they send is recursively string-encoded.
+    if type(stringy_json) == dict:
+        for key in stringy_json:
+            stringy_json[key] = decode_json(stringy_json[key])
+    elif type(stringy_json) == list:
+        for i in range(len(stringy_json)):
+            stringy_json[i] = decode_json(stringy_json[i])
+    elif type(stringy_json) in (str,unicode):
+        if len(stringy_json) > 0 and stringy_json[0] in ["[","{",u"[",u"{"]:
+            return decode_json(json.loads(stringy_json))
+    return stringy_json
+
+
+if __name__ == '__main__':
+    main()

From 1e9d78672fd6b38b62df48553c464664f6393e70 Mon Sep 17 00:00:00 2001
From: Serdar Tumgoren <zstumgoren@gmail.com>
Date: Fri, 2 Dec 2016 20:58:27 -0800
Subject: [PATCH 2/3] DC: remove unused var in bill dump script

---
 scripts/dc/dump_bill_json.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/dc/dump_bill_json.py b/scripts/dc/dump_bill_json.py
index e53c1d5071..2c2552633d 100644
--- a/scripts/dc/dump_bill_json.py
+++ b/scripts/dc/dump_bill_json.py
@@ -22,7 +22,6 @@ def main():
     bill_params = { "legislationId" : bill_id }
     bill_info = requests.post(bill_url, headers=headers, data=json.dumps(bill_params))
     bill_info = decode_json(bill_info.json()["d"])["data"]
-    leg_info = bill_info["Legislation"][0]
     pprint.pprint(bill_info)
 
 def decode_json(stringy_json):

From b83870f237d047af260513b149d3cda273f63943 Mon Sep 17 00:00:00 2001
From: Serdar Tumgoren <zstumgoren@gmail.com>
Date: Fri, 2 Dec 2016 21:47:47 -0800
Subject: [PATCH 3/3] Add mkdir_p utility and update DC helper script to dump
 raw json to data dir

---
 openstates/utils/__init__.py |  1 +
 openstates/utils/dir.py      | 11 +++++++++++
 scripts/dc/dump_bill_json.py |  8 ++++++++
 3 files changed, 20 insertions(+)
 create mode 100644 openstates/utils/dir.py

diff --git a/openstates/utils/__init__.py b/openstates/utils/__init__.py
index 58fd21e5d7..68029c5388 100644
--- a/openstates/utils/__init__.py
+++ b/openstates/utils/__init__.py
@@ -1,4 +1,5 @@
 from .lxmlize import LXMLMixin
+from .dir import mkdir_p
 import re
 
 
diff --git a/openstates/utils/dir.py b/openstates/utils/dir.py
new file mode 100644
index 0000000000..00e48809b3
--- /dev/null
+++ b/openstates/utils/dir.py
@@ -0,0 +1,11 @@
+import errno
+import os
+
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError as exc:  # Python >2.5
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
diff --git a/scripts/dc/dump_bill_json.py b/scripts/dc/dump_bill_json.py
index 2c2552633d..03db2f7ffa 100644
--- a/scripts/dc/dump_bill_json.py
+++ b/scripts/dc/dump_bill_json.py
@@ -11,6 +11,8 @@
 
 import requests
 
+from openstates.utils import mkdir_p
+
 def main():
     headers = {"Content-Type":"application/json"}
     bill_url = "http://lims.dccouncil.us/_layouts/15/uploader/AdminProxy.aspx/GetPublicData"
@@ -22,7 +24,13 @@ def main():
     bill_params = { "legislationId" : bill_id }
     bill_info = requests.post(bill_url, headers=headers, data=json.dumps(bill_params))
     bill_info = decode_json(bill_info.json()["d"])["data"]
+    output_dir = 'data/dc/bills_raw/'
+    outfile = "".join([output_dir, "{}.json".format(bill_id)])
+    mkdir_p(output_dir)
+    with open(outfile, 'w') as out:
+        json.dump(bill_info, out, sort_keys=True, indent=4, separators=(',', ': '))
     pprint.pprint(bill_info)
+    print("\nRaw bill data saved: {}\n".format(outfile))
 
 def decode_json(stringy_json):
     #the "json" they send is recursively string-encoded.