Commit `1627615c29`

1627615c291024f159f6699768fbc13662e83bbd

Verified · cmc

cmc <hello@cleberg.net> · 2024-01-24 19:36 UTC

update path and add method to delete extra header rows

Layout: unified · split

ingest.py +7 −3

		@@ -4,7 +4,7 @@ import sqlite3
4	4	import os
5	5
6	6	# Create the database file
7		connection = sqlite3.connect('./raw_data/ingress.db')
	7	connection = sqlite3.connect('./raw_data/test.db')
8	8
9	9	# Creating a cursor object to execute SQL queries
10	10	cursor = connection.cursor()
		@@ -34,14 +34,14 @@ create_table = '''CREATE TABLE incidents(
34	34	cursor.execute(create_table)
35	35
36	36	# Point to the data directory
37		directory = os.fsencode("../raw_data/")
	37	directory = os.fsencode("./raw_data/")
38	38
39	39	# Loop through all raw data files
40	40	for file in os.listdir(directory):
41	41	filename = os.fsdecode(file)
42	42	if filename.endswith(".csv"):
43	43	# Opening the file
44		file = open("../raw_data/" + filename)
	44	file = open("./raw_data/" + filename)
45	45
46	46	# Reading the contents of the file
47	47	contents = csv.reader(file)
		@@ -58,6 +58,10 @@ for file in os.listdir(directory):
58	58	else:
59	59	continue
60	60
	61	# Delete extra copies of the header row that were inserted
	62	delete_headers = "DELETE FROM incidents WHERE rb = 'RB Number'"
	63	cursor.execute(delete_headers)
	64
61	65	# Test query to see if the data loaded
62	66	select_all = "SELECT * FROM incidents"
63	67	rows = cursor.execute(select_all).fetchall()

Commit 1627615c29