使用Python将CSV文件导入到sqlite3数据库表中
我有一个CSV文件，我想用Python将这个文件批量导入到我的sqlite3数据库中。 该命令是“.import …..”。 但似乎不能像这样工作。 任何人都可以给我一个例子，如何在sqlite3中做到这一点？ 我正在使用Windows以防万一。 谢谢
import csv, sqlite3 con = sqlite3.connect(":memory:") cur = con.cursor() cur.execute("CREATE TABLE t (col1, col2);") # use your column names here with open('data.csv','rb') as fin: # `with` statement available in 2.5+ # csv.DictReader uses first line in file for column headings by default dr = csv.DictReader(fin) # comma is default delimiter to_db = [(i['col1'], i['col2']) for i in dr] cur.executemany("INSERT INTO t (col1, col2) VALUES (?, ?);", to_db) con.commit() con.close() 创build一个到磁盘上文件的sqlite连接是读者的一个练习…但是现在有一个双线程可以通过pandas库
df = pandas.read_csv(csvfile) df.to_sql(table_name, conn, if_exists='append', index=False)
.import命令是sqlite3命令行工具的一个function。 要在Python中完成，只需使用Python提供的任何工具（如csv模块） ，并按照惯例插入数据即可加载数据。
这样，您也可以控制插入的types，而不是依赖sqlite3看似没有logging的行为。
我的2美分（更通用）：
import csv, sqlite3 import logging def _get_col_datatypes(fin): dr = csv.DictReader(fin) # comma is default delimiter fieldTypes = {} for entry in dr: feildslLeft = [f for f in dr.fieldnames if f not in fieldTypes.keys()] if not feildslLeft: break # We're done for field in feildslLeft: data = entry[field] # Need data to decide if len(data) == 0: continue if data.isdigit(): fieldTypes[field] = "INTEGER" else: fieldTypes[field] = "TEXT" # TODO: Currently there's no support for DATE in sqllite if len(feildslLeft) > 0: raise Exception("Failed to find all the columns data types - Maybe some are empty?") return fieldTypes def escapingGenerator(f): for line in f: yield line.encode("ascii", "xmlcharrefreplace").decode("ascii") def csvToDb(csvFile, outputToFile = False): # TODO: implement output to file with open(csvFile,mode='r', encoding="ISO-8859-1") as fin: dt = _get_col_datatypes(fin) fin.seek(0) reader = csv.DictReader(fin) # Keep the order of the columns name just as in the CSV fields = reader.fieldnames cols = [] # Set field and type for f in fields: cols.append("%s %s" % (f, dt[f])) # Generate create table statement: stmt = "CREATE TABLE ads (%s)" % ",".join(cols) con = sqlite3.connect(":memory:") cur = con.cursor() cur.execute(stmt) fin.seek(0) reader = csv.reader(escapingGenerator(fin)) # Generate insert statement: stmt = "INSERT INTO ads VALUES(%s);" % ','.join('?' * len(cols)) cur.executemany(stmt, reader) con.commit() return con
非常感谢bernie的回答 ！ 不得不调整一下 – 这是对我有用：
import csv, sqlite3 conn = sqlite3.connect("pcfc.sl3") curs = conn.cursor() curs.execute("CREATE TABLE PCFC (id INTEGER PRIMARY KEY, type INTEGER, term TEXT, definition TEXT);") reader = csv.reader(open('PC.txt', 'r'), delimiter='|') for row in reader: to_db = [unicode(row[0], "utf8"), unicode(row[1], "utf8"), unicode(row[2], "utf8")] curs.execute("INSERT INTO PCFC (type, term, definition) VALUES (?, ?, ?);", to_db) conn.commit()
我的文本文件（PC.txt）看起来像这样：
1 | Term 1 | Definition 1 2 | Term 2 | Definition 2 3 | Term 3 | Definition 3
#!/usr/bin/python # -*- coding: utf-8 -*- import sys, csv, sqlite3 def main(): con = sqlite3.connect(sys.argv[1]) # database file input cur = con.cursor() cur.executescript(""" DROP TABLE IF EXISTS t; CREATE TABLE t (COL1 TEXT, COL2 TEXT); """) # checks to see if table exists and makes a fresh table. with open(sys.argv[2], "rb") as f: # CSV file input reader = csv.reader(f, delimiter=',') # no header information with delimiter for row in reader: to_db = [unicode(row[0], "utf8"), unicode(row[1], "utf8")] # Appends data from CSV file representing and handling of text cur.execute("INSERT INTO neto (COL1, COL2) VALUES(?, ?);", to_db) con.commit() con.close() # closes connection to database if __name__=='__main__': main()
你可以有效地使用blaze和odo来做到这一点
import blaze csv_path = 'data.csv' bz.odo(csv_path, 'sqlite:///data.db::data')
Odo会将csv文件存储到模式data下的data.db （sqlite数据库）中
或者你直接使用odo ，没有blaze 。 无论哪种方式都很好。 阅读本文档
