sqlite3 scrapy
tokauf at gmail.com
tokauf at gmail.com
Thu Jul 7 10:04:45 EDT 2016
Hi,
I have a little scrapy-script: scrapy gets the links from a webpage; this works fine. Then I want to write these links in a sqlite3-table. There is no error-note. But in the database are not any records. What is the problem here? My code:
# -*- coding: utf-8 -*-
2
3 import scrapy
4 import sqlite3
5
6 class MySpider(scrapy.Spider):
7
8 name = "frisch2"
9 allowed_domains = ["frischblau.de"]
10 start_urls = ["http://www.frischblau.de/",]
11
12 def parse(self, response):
13
14 for href in response.xpath("//ul/li/a/@href"):
15
16 url = response.urljoin(href.extract())
17 yield scrapy.Request(url, callback=self.parse_dir_contents)
18
19 def parse_dir_contents(self, response):
20
21 sql = u' '
22 conn = sqlite3.connect('frisch.db')
23 cur = conn.cursor()
24 counter = 3;
25
26 try:
27 cur.execute("SELECT SQLITE_VERSION()")
28 print "-> SQLite version: %s" % cur.fetchone()
29
30 item = response.xpath('//a/@href').extract()
31 for el in item:
32
33 #print("----------> ", type(el))
34 sql = "INSERT INTO Frisch VALUES(" + unicode(counter) + ", " + "'" + el.strip() + "');"
35
36 print (sql)
37 cur.execute(sql)
38 conn.commit
39 counter = int(counter)
40 counter += 1
41
42 except sqlite3.Error as e:
43 print "Error %s:" % e.args[0]
44 conn.close()
45
Thanks for help.
o-o
Thomas
More information about the Python-list
mailing list