Commit 1627615c29
Verified · cmc
Layout: unified · split
ingest.py +7 −3
| @@ -4,7 +4,7 @@ import sqlite3 | |||
| 4 | import os | 4 | import os |
| 5 | 5 | ||
| 6 | # Create the database file | 6 | # Create the database file |
| 7 | connection = sqlite3.connect('./raw_data/ingress.db') | 7 | connection = sqlite3.connect('./raw_data/test.db') |
| 8 | 8 | ||
| 9 | # Creating a cursor object to execute SQL queries | 9 | # Creating a cursor object to execute SQL queries |
| 10 | cursor = connection.cursor() | 10 | cursor = connection.cursor() |
| @@ -34,14 +34,14 @@ create_table = '''CREATE TABLE incidents( | |||
| 34 | cursor.execute(create_table) | 34 | cursor.execute(create_table) |
| 35 | 35 | ||
| 36 | # Point to the data directory | 36 | # Point to the data directory |
| 37 | directory = os.fsencode("../raw_data/") | 37 | directory = os.fsencode("./raw_data/") |
| 38 | 38 | ||
| 39 | # Loop through all raw data files | 39 | # Loop through all raw data files |
| 40 | for file in os.listdir(directory): | 40 | for file in os.listdir(directory): |
| 41 | filename = os.fsdecode(file) | 41 | filename = os.fsdecode(file) |
| 42 | if filename.endswith(".csv"): | 42 | if filename.endswith(".csv"): |
| 43 | # Opening the file | 43 | # Opening the file |
| 44 | file = open("../raw_data/" + filename) | 44 | file = open("./raw_data/" + filename) |
| 45 | 45 | ||
| 46 | # Reading the contents of the file | 46 | # Reading the contents of the file |
| 47 | contents = csv.reader(file) | 47 | contents = csv.reader(file) |
| @@ -58,6 +58,10 @@ for file in os.listdir(directory): | |||
| 58 | else: | 58 | else: |
| 59 | continue | 59 | continue |
| 60 | 60 | ||
| 61 | # Delete extra copies of the header row that were inserted | ||
| 62 | delete_headers = "DELETE FROM incidents WHERE rb = 'RB Number'" | ||
| 63 | cursor.execute(delete_headers) | ||
| 64 | |||
| 61 | # Test query to see if the data loaded | 65 | # Test query to see if the data loaded |
| 62 | select_all = "SELECT * FROM incidents" | 66 | select_all = "SELECT * FROM incidents" |
| 63 | rows = cursor.execute(select_all).fetchall() | 67 | rows = cursor.execute(select_all).fetchall() |